#coding:utf-8
import random
import urllib2
from urllib2 import URLError
from urllib2 import HTTPError
import os
from utils import clean_html,segment
from sklearn import linear_model
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
import joblib
import sys
from crawurl import http_client

    
def kwdetect_html(html):
    if not html:
        return ''
    x_predict = []
    vector = joblib.load('/opt/disk2/var/www/myblog/mlmodels/tfidf.m')
    model = joblib.load('/opt/disk2/var/www/myblog/mlmodels/clf.m')
    content = clean_html(html)
    seg = segment(content)
    x_predict.append(seg)
    x_predict = vector.transform(x_predict)
    predict = model.predict(x_predict)
    return predict

def kwdetect_url(url):
    if not url:
        return ''
    content = http_client(url)
    predict = kwdetect_html(content)
    return predict

def kwdetect_urls(url_list):
    predict = []
    for url in url_list:
        p = kwdetect_url(url)
        predict.extend(p)
    return predict
        


    
